// pipeline_mongodb.go

package pipeline

import (
	"context"
	"net/url"
	"time"

	"gitee.com/congqian/dense-spider/core/common/com_interfaces"
	"gitee.com/congqian/dense-spider/core/common/page_items"
	"go.mongodb.org/mongo-driver/mongo"
	"go.mongodb.org/mongo-driver/mongo/options"
)

type MDocument struct {
	Title        string
	Domain       string
	Content      string
	Url          string
	Created      time.Time
	DocPublished time.Time
}
type PipelineMongodb struct {
}

func NewPipelineMongodb() *PipelineMongodb {
	return &PipelineMongodb{}
}

func (this *PipelineMongodb) Process(items *page_items.PageItems, t com_interfaces.Task) {
	domain := ""
	url, err := url.Parse(items.GetRequest().GetUrl())
	if err != nil {
		domain = ""
	} else {
		domain = url.Host
	}

	content, ok := items.GetItem("Content")
	if !ok {
		content = ""
	}
	docPublished := time.Now()

	title, ok := items.GetItem("Title")
	if !ok {
		title = ""
	}
	if title == "" || content == "" {
		return
	}
	doc := &MDocument{Domain: domain, Url: items.GetRequest().GetUrl(), DocPublished: docPublished, Title: title, Content: content, Created: time.Now()}
	// 连接数据库
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	client, err := mongo.Connect(ctx, options.Client().ApplyURI("mongodb://localhost:27017"))
	if err != nil {
		panic(err)
	}
	defer client.Disconnect(ctx)

	// 获取数据库,获取集合
	c := client.Database("go_spider").Collection("Document")

	// 存储数据

	_, err = c.InsertOne(ctx, doc)

	if err != nil {
		println(err.Error())
		panic(err)
	}
}
func main() {

}
